package com.terren.spider.core.html.pipeline;

import java.util.List;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;

import com.terren.spider.core.html.configuration.Configuration;
import com.terren.spider.core.html.scheme.ICustomScheme;
import com.terren.spider.entity.biz.ArticleSearch;
import com.terren.spider.entity.common.SpiderLog;
import com.terren.spider.entity.config.SysConfig;
import com.terren.spider.entity.core.Entry;
import com.terren.spider.util.common.BeanUtil;
import com.terren.spider.util.common.StringUtil;
import com.terren.spider.util.common.TextExtract;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;

/**
 * 搜索页的pipeline 继承 ResultItemsCollectorPipeline
 * 
 * @author chengyu
 *
 * @param <T>
 */
public class HtmlArticleSearchDBCollectorPipeline<T> extends BasicHtmlCollectorPipeline<T> {

	public HtmlArticleSearchDBCollectorPipeline(Configuration configuration, Class<T> clazz)
			throws Exception {
		super(configuration, clazz);
	}

	public synchronized void process(ResultItems resultItems, Task task) {

		/**
		 * 目标页面 即搜索到的页面
		 */
		if (resultItems.get(SysConfig.COLLECT_ITEM_KEY) instanceof Page) {
			Entry entry = configuration.getEntry();
			try {
				Page page = resultItems.get(SysConfig.COLLECT_ITEM_KEY);
				Integer totalPage = resultItems.get(SysConfig.COLLECT_TOTAL_TARGET_URL);
				/**
				 * 分析前处理
				 */
				for (ICustomScheme	scheme : configuration.getSchemes()) {
					if (BeanUtil.includeMethod("preParse", scheme.getClass())) {
						scheme.preParse(page, configuration);
					}
				}
				/**
				 * 是否为需要过滤掉的文章
				 */
				if (isExcludeContent(page.getRawText(), entry)) {
					return;
				}
				T pojo = null;
				String url = resultItems.get(SysConfig.COLLECT_ITEM_LAST_REDIRECT_URL) != null
						? resultItems.get(SysConfig.COLLECT_ITEM_LAST_REDIRECT_URL)
						: page.getUrl().toString();
				List<Entry> entryList = entry.getSubEntry();
				Entry subEntry = findSubEntry(url, entryList);
				/**
				 * 如果包含配置的目标页entry
				 */
				if (null != subEntry) {
					pojo = parseUtil.parse(page,subEntry,clazz);
				}
				/**
				 * 如果不包含配置的目标页entry
				 */
				else {
					pojo = parseUtil.parse(page,entry,clazz);
				}
				if (pojo instanceof ArticleSearch) {
					if (null == ((ArticleSearch) pojo).getTitle()) {
						String title = resultItems.get(SysConfig.COLLECT_ITEM_TITLE_KEY) != null
								? resultItems.get(SysConfig.COLLECT_ITEM_TITLE_KEY)
								: "";
						((ArticleSearch) pojo).setTitle(title);
					}
					if (null == ((ArticleSearch) pojo).getPubTime() && null==((ArticleSearch) pojo).getPubTimeStr() ) {
						String pubTimeStr = resultItems.get(SysConfig.COLLECT_ITEM_PUBTIME_KEY) != null
								? resultItems.get(SysConfig.COLLECT_ITEM_PUBTIME_KEY)
								: "";
						((ArticleSearch) pojo).setPubTimeStr(pubTimeStr);
					}
					if (null == ((ArticleSearch) pojo).getNewsSource()) {
						String newsSource = resultItems.get(SysConfig.COLLECT_ITEM_SOURCE_KEY) != null
								? resultItems.get(SysConfig.COLLECT_ITEM_SOURCE_KEY)
								: "";
					   ((ArticleSearch) pojo).setNewsSource(newsSource);		
					}
					if (null == ((ArticleSearch) pojo).getAuthor()) {
						String author = resultItems.get(SysConfig.COLLECT_ITEM_AUTHOR_KEY) != null
								? resultItems.get(SysConfig.COLLECT_ITEM_AUTHOR_KEY)
								: "";
					   ((ArticleSearch) pojo).setAuthor(author);		
					}
					if (null == ((ArticleSearch) pojo).getAuthorUrl()) {
						String authorUrl = resultItems.get(SysConfig.COLLECT_ITEM_AUTHOR_URL_KEY) != null
								? resultItems.get(SysConfig.COLLECT_ITEM_AUTHOR_URL_KEY)
								: "";
					   ((ArticleSearch) pojo).setAuthorUrl(authorUrl);	
					}
					String html = ((ArticleSearch) pojo).getHtml();
					/**
					 * 是否为需要过滤掉的文章
					 */
					if (isExcludeContent(html, entry)) {
						return;
					}
					if (null != html && ((ArticleSearch) pojo).getContent() == null) {
						String content = TextExtract.parse(html);
						if (content == null || "".equals(content)) {
							return;
						}
						((ArticleSearch) pojo).setContent(content);
						
					} else if (((ArticleSearch) pojo).getContent() == null) {
						return;//丢弃掉没有文章内容的page
					}
					
					if (null == ((ArticleSearch) pojo).getAuthor()) {
						// 保存作者
						((ArticleSearch) pojo).setAuthor(StringUtil.getParseStr(html, "作者"));
					}
					if (null ==((ArticleSearch) pojo).getOriginalSource() ) {
						// 保存新闻来源
						((ArticleSearch) pojo).setOriginalSource(StringUtil.getParseStr(html, "来源"));
					}
					((ArticleSearch) pojo).setBeginTime(entry.getBeginTime() != null ? entry.getBeginTime() : "");
					((ArticleSearch) pojo).setEndTime(entry.getEndTime() != null ? entry.getEndTime() : "");
					((ArticleSearch) pojo).setTaskId(entry.getTaskId());
					((ArticleSearch) pojo).setKeyword(entry.getKeyword());
					((ArticleSearch) pojo).setSourceId(entry.getSourceId());
					((ArticleSearch) pojo).setUrl(url);
				}
				
				
				this.collectItems.add(pojo);
				saveToDB(this.collectItems, totalPage);
			} catch (Exception e) {
				log.error("[entry:"+entry.getEntryId()+"]save to db error:"+e.getMessage(),e);
				try {
					SpiderLog spiderLog = new SpiderLog();
					spiderLog.setMessage("[entry:"+entry.getEntryId()+"]save to db error:"+e.getMessage());
					spiderLog.setStatus(0);
					spiderLogService.addSpiderLog(spiderLog);
				} catch (Exception e1) {
					log.error("[entry:"+entry.getEntryId()+"]save to spider log error:"+e.getMessage(),e1);
				}
			}
		}
		// collector.add(resultItems);
	}

	/**
	 * 是否为需要过滤掉的文章
	 * 
	 * @param content
	 * @param entry
	 * @return
	 */
	protected boolean isExcludeContent(String content, Entry entry) {
		if (null != entry.getExcludeKeyword()&&null!=content) {
			String[] excludeKey = entry.getExcludeKeyword().split(",");
			for (String key : excludeKey) {
				if (StringUtils.contains(content, key)&& !key.equals("")) {
					log.info("is contains exclude keyword" + key);
					return true;
				}
			}
		}
		return false;
	}


	protected Entry findSubEntry(String url, List<Entry> entryList) {
		if (CollectionUtils.isNotEmpty(entryList)) {
			for (Entry entry : entryList) {
				if (null != entry.getUrlRegex() && !"".equals(entry.getUrlRegex())) {
					String[] urlRegexList = entry.getUrlRegex().split(SysConfig.URL_SPLIT);
					for (String urlreg : urlRegexList) {
						if (StringUtil.match(url, urlreg)) {
							return entry;
						}
					}
				}
			}
		}
		return null;
	}
}
